home *** CD-ROM | disk | FTP | other *** search
# Source Generated with Decompyle++ # File: in.pyo (Python 2.5) from __future__ import with_statement import sys import traceback import re import struct import logging import socks import socket import asynchat import urllib import urllib2 import urlparse import httplib import httplib2 import cookielib from httplib import HTTPConnection from httplib import NotConnected import primitives import proxy_settings from Events import EventMixin from callbacks import callsback log = logging.getLogger('util.net') default_chunksize = 4096 def get_ips_s(hostname = ''): if not hostname: pass return socket.gethostbyname_ex(socket.gethostname())[2] def get_ips(hostname = ''): return [ socket.inet_aton(ip) for ip in get_ips_s(hostname) ] myips = get_ips def myip(): return myips()[0] def ip_from_bytes(bytes): return socket.inet_ntoa(bytes) class FileChunker: def __init__(self, fileobj, chunksize = default_chunksize, close_when_done = False, progress_cb = (lambda bytes: pass), bytecounter = None): self.fileobj = fileobj self.chunksize = chunksize self.close_when_done = close_when_done if bytecounter is None: bytecounter = fileobj.tell self.total = bytecounter() self.progress_cb = progress_cb self.cancelled = False def more(self): try: data_read = self.fileobj.read(self.chunksize) except ValueError: try: self.fileobj.close() except: pass return '' sz = len(data_read) if sz == 0 and self.close_when_done: self.fileobj.close() self.total += sz self.progress_cb(self.total) return data_read def tofile(cls, sourcefile, outfile, progress_callback = (lambda : pass), bytecounter = None): gen = cls.tofile_gen(sourcefile, outfile, progress_callback, bytecounter) gen.next() try: gen.next() except StopIteration: pass tofile = classmethod(tofile) def tofile_gen(cls, sourcefile, outfile, progress_callback = (lambda : pass), bytecounter = None): fc = cls(sourcefile, close_when_done = True, bytecounter = bytecounter) yield fc chunk = fc.more() bytes_written = 0 write = outfile.write tell = outfile.tell more = fc.more while chunk and not (fc.cancelled): write(chunk) bytes_written += len(chunk) progress_callback(tell()) chunk = more() outfile.close() tofile_gen = classmethod(tofile_gen) def httpjoin(base, path, keepquery = False): if path.startswith('http'): return path else: joined = urlparse.urljoin(base, path) if not keepquery: parsed = list(urlparse.urlparse(joined)) parsed[4] = '' return urlparse.urlunparse(parsed) else: return joined class UrlQuery(str): def parse(cls, url, parse_query = True): (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url) if parse_query: querymap = { } for elem in query.split('&'): (key, value) = None if '=' in elem else (elem, True) querymap[key] = value query = querymap return dict(scheme = scheme, netloc = netloc, path = path, params = params, query = query, fragment = fragment) parse = classmethod(parse) def __new__(cls, link, d = { }, **kwargs): if link.endswith('?'): link = link[:-1] if '?' in link: joiner = '&' else: joiner = '?' return str.__new__(cls, ''.join([ link, joiner, WebFormData(d = d, **kwargs)])) class WebFormData(str): def __new__(cls, d = { }, **kwargs): if d and kwargs: kwargs.update(d) elif not kwargs: pass kwargs = d base = urllib.urlencode(kwargs) return str.__new__(cls, base) def int_to_ip(s, byteorder = '<'): return '.'.join((lambda .0: for c in .0: str(ord(c)))(struct.pack(byteorder + 'I', int(s)))) spacify_pattern = re.compile('( {2,})') def spacify_repl(m): l = len(m.group()) if l == 2: return ' ' else: return ' ' + ''.join([ ' '] * (l - 2)) + ' ' def spacify(s): return spacify_pattern.sub(spacify_repl, s) urlregex = re.compile("([A-Za-z][A-Za-z0-9+.-]{1,120}:[A-Za-z0-9/](([A-Za-z0-9$_.+!*,;/?:@&~=-])|%[A-Fa-f0-9]{2}){1,333}(#([a-zA-Z0-9][a-zA-Z0-9$_.+!*,;/?:@&~=%-']{0,1000}))?)[^\\. <]") TLDs = [ 'arpa', 'root', 'aero', 'asia', 'biz', 'com', 'coop', 'edu', 'gov', 'info', 'int', 'museum', 'name', 'net', 'org', 'pro', 'ac', 'ad', 'ae', 'af', 'ag', 'ai', 'al', 'am', 'an', 'ao', 'aq', 'ar', 'as', 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd', 'be', 'bf', 'bg', 'bh', 'bi', 'bj', 'bm', 'bn', 'bo', 'br', 'bs', 'bt', 'bv', 'bw', 'by', 'bz', 'ca', 'cc', 'cd', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'cm', 'cn', 'co', 'cr', 'cu', 'cv', 'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm', 'do', 'dz', 'ec', 'ee', 'eg', 'er', 'es', 'et', 'eu', 'fi', 'fj', 'fk', 'fm', 'fo', 'fr', 'ga', 'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'gm', 'gn', 'gp', 'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk', 'hm', 'hn', 'hr', 'ht', 'hu', 'id', 'ie', 'il', 'im', 'in', 'io', 'iq', 'ir', 'is', 'it', 'je', 'jm', 'jo', 'jp', 'ke', 'kg', 'kh', 'ki', 'km', 'kn', 'kp', 'kr', 'kw', 'ky', 'kz', 'la', 'lb', 'lc', 'li', 'lk', 'lr', 'ls', 'lt', 'lu', 'lv', 'ly', 'ma', 'mc', 'md', 'me', 'mg', 'mh', 'mk', 'ml', 'mm', 'mn', 'mo', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu', 'mv', 'mw', 'mx', 'my', 'mz', 'na', 'nc', 'ne', 'nf', 'ng', 'ni', 'nl', 'no', 'np', 'nr', 'nu', 'nz', 'om', 'pa', 'pe', 'pf', 'pg', 'ph', 'pk', 'pl', 'pm', 'pn', 'pr', 'ps', 'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru', 'rw', 'sa', 'sb', 'sc', 'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl', 'sm', 'sn', 'so', 'sr', 'st', 'su', 'sv', 'sy', 'sz', 'tc', 'td', 'tf', 'tg', 'th', 'tj', 'tk', 'tl', 'tm', 'tn', 'to', 'tp', 'tr', 'tt', 'tv', 'tw', 'tz', 'ua', 'ug', 'uk', 'um', 'us', 'uy', 'uz', 'va', 'vc', 've', 'vg', 'vi', 'vn', 'vu', 'wf', 'ws', 'ye', 'yt', 'yu', 'za', 'zm', 'zw'] domains = '(?:%s)' % '|'.join(TLDs) email_regex_string = '(([a-zA-Z0-9_][a-zA-Z0-9_\\-\\.]*)(\\+[a-zA-Z0-9_\\-\\.]+)?@(([a-zA-Z0-9\\-_]+\\.?)*[a-zA-Z]{1,4}))' email_regex = re.compile(email_regex_string) email_wholestring_regex = re.compile('^' + email_regex_string + '$') is_email = primitives.ischeck((lambda s: bool(email_wholestring_regex.match(s)))) class EmailAddress(tuple): def __new__(cls, addr, default_domain = sentinel): try: (name, label, domain) = parse_email(addr) except: if default_domain is sentinel: raise else: (name, label, domain) = parse_email(addr + '@' + default_domain) return tuple.__new__(cls, (name, label, domain)) def name(self): return self[0] name = property(name) def label(self): return self[1] label = property(label) def domain(self): return self[2] domain = property(domain) def __str__(self): if self.label: return '%s+%s@%s' % self else: return '%s@%s' % (self.name, self.domain) def __repr__(self): return '<EmailAddress %s>' % (self,) def parse_email(s): match = email_wholestring_regex.match(s) (wholething, user, lbl, dom, __) = match.groups() if lbl: lbl = lbl.strip('+') else: lbl = '' return (user, lbl, dom) protocols = 'ftp|https?|gopher|msnim|icq|telnet|nntp|aim|file|svn|svn+(?:\\w)+' linkify_url_pattern = re.compile('(?=[a-zA-Z0-9]) # Must start correctly\n ((?: # Match the leading part (proto://hostname, or just hostname)\n (?P<protocol>\n (?:%s) # protocol\n :// # ://\n (?: # Optional \'username:password@\'\n \\w+ # username\n (?::\\w+)? # optional :password\n @ # @\n )?)? #\n (?P<hostname>[-\\w]+(?:\\.\\w[-\\w]*)*) # hostname (sub.example.com)\n )? #\n (?::\\d+)? # Optional port number\n (?: # Rest of the URL, optional\n /? # Start with \'/\'\n [^.!,?;:"<>\\[\\]{}\\s\\x7F-\\xFF]* # Can\'t start with these\n (?: #\n [.!,?;:]+ # One or more of these\n [^.!,?;:"<>\\[\\]{}\\s\\x7F-\\xFF]+ # Can\'t finish with these\n #\'" # # or \' or "\n )* #\n )?) #\n ' % protocols, re.VERBOSE) def isurl(text): m = linkify_url_pattern.match(text) if not m: return False protocol = m.group('protocol') host = m.group('hostname') if host is not None: if protocol is None: myTLDs = None if '.' in host else [ host] if len(myTLDs) < 2 or myTLDs[-1] not in TLDs: return False return True def _dolinkify(text): def repl(m): protocol = m.group('protocol') host = m.group('hostname') url = m.group() after = '' if url.endswith(')') and '(' not in url: url = url[:-1] after = ')' if host is not None: if protocol is None: myTLDs = None if '.' in host else [ host] if len(myTLDs) < 2 or myTLDs[-1] not in TLDs: return url + after href = None if protocol is None else url return '<a href="%s">%s</a>' % (href, url) + after text = linkify_url_pattern.sub(repl, text) return text def linkify(text): if not re.search('<.*>', text): return _dolinkify(text) else: lines = [] for line in re.split('(<.*?>)', text): if not re.match('<.*?>', line): line = _dolinkify(line) lines.append(line) return ''.join(lines) class QueueableMixin(object): def __init__(self): object.__init__(self) self._on_queue = False def queue(self): if not self._on_queue: self._queue() self._on_queue = True def unqueue(self): if self._on_queue: self._unqueue() self._on_queue = False class ProducerQueuable(QueueableMixin): def __init__(self, sck): QueueableMixin.__init__(self) self.sck = sck def _queue(self): self.sck.push_with_producer(self) def _unqueue(self): try: self.sck.producer_fifo.remove(self) except ValueError: pass class RoundRobinProducer(ProducerQueuable): def __init__(self, sck): ProducerQueuable.__init__(self, sck) self.list = [] def add(self, prod): try: if not callable(prod.more): raise AssertionError('Producers must have a "more" method') except: traceback.print_exc() raise self.unqueue() self.list.append(prod) self.queue() def more(self): self._on_queue = True d = None l = self.list prod = None while not d and l: prod = l.pop(0) d = prod.more() if d: l.append(prod) else: self.unqueue() if self.list: self.queue() return d class PriorityProducer(ProducerQueuable): def __init__(self, sck): ProducerQueuable.__init__(self, sck) self.high = [] self.mid = [] self.low = [] def add(self, prod, pri = 'mid'): self.unqueue() getattr(self, pri).append(prod) self.queue() def more(self): self._on_queue = True d = None for l in (self.high, self.mid, self.low): if not l: continue while not d and l: prod = l.pop(0) d = prod.more() if d: l.insert(0, prod) break continue return d class HTTPConnProgress(HTTPConnection): def send_file_cb(self, fileobj, progress_cb, blocksize = default_chunksize, progressDelta = 0): if self.sock is None: if self.auto_open: self.connect() else: raise NotConnected() if self.debuglevel > 0: print 'sending contents of', fileobj try: read = fileobj.read sendall = self.sock.sendall chunk = read(blocksize) total = 0 while chunk: total += len(chunk) sendall(chunk) progress_cb(total - progressDelta) chunk = read(blocksize) except socket.error: v = None if v[0] == 32: self.close() raise class SocketEventMixin(EventMixin): events = EventMixin.events | set(('connected', 'connection_failed', 'socket_error', 'socket_closed')) def post_connect_error(self, e = None): self.event('socket_error') self.post_connect_disconnect() def post_connect_expt(self): self.event('socket_error') self.post_connect_disconnect() def post_connect_disconnect(self): self.close() self.event('socket_closed') def post_connect_close(self): self.close() self.event('socket_closed') def reassign(self): self.handle_expt = self.post_connect_expt self.handle_error = self.post_connect_error self.handle_close = self.post_connect_close self.do_disconnect = self.post_connect_disconnect def build_cookie(name, value, version = 0, domain = sentinel, port = sentinel, path = sentinel, secure = False, expires = None, discard = False, comment = None, comment_url = None, rest = { 'httponly': None }, rfc2109 = False): if domain is sentinel: domain = None domain_specified = False domain_initial_dot = False else: domain_specified = True domain_initial_dot = domain.startswith('.') if port is sentinel: port = None port_specified = False else: port_specified = True if path is sentinel: path = None path_specified = False else: path_specified = True return cookielib.Cookie(**locals()) def GetSocketType(): d = GetProxyInfo() if d: return socks.socksocket else: return socket.socket NONE = 'NONPROX' SYSDEFAULT = 'SYSPROX' CUSTOM = 'SETPROX' def GetProxyInfo(): ps = proxy_settings try: pd = ps.get_proxy_dict() except Exception: e = None print >>sys.stderr, 'No proxies because: %r' % e pd = { } get = pd.get proxytype = get('proxytype') port = get('port') try: port = int(port) except: port = None addr = get('addr') username = get('username') password = get('password') override = get('override') try: override = int(override) except: if override not in (SYSDEFAULT, CUSTOM, NONE): override = SYSDEFAULT if override: override = CUSTOM else: override = SYSDEFAULT if override == NONE: return { } elif override == SYSDEFAULT: px = urllib._getproxies() if not px: return { } url = px.get('http', None) if url is None: return { } url = urlparse.urlparse(url) if not url.hostname: pass addr = '' if not addr: return { } if not url.port: pass port = 80 if not url.username and username: pass username = None if not url.password and password: pass password = None proxytype = 'http' if all((type, port, addr)): proxytype = getattr(socks, ('proxy_type_%s' % proxytype).upper(), None) return dict(addr = addr, port = port, username = username, password = password, proxytype = proxytype) else: return { } def GetProxyInfoHttp2(): i = GetProxyInfo() if not i: return None return httplib2.ProxyInfo(proxy_type = i['proxytype'], proxy_host = i['addr'], proxy_port = i['port'], proxy_user = i['username'], proxy_pass = i['password'], proxy_rdns = True) def getproxies_digsby(): pinfo = GetProxyInfo() proxies = { } if pinfo.get('username', None) and pinfo.get('password', None): unpw = '%s:%s@' % (pinfo['username'], pinfo['password']) else: unpw = '' if pinfo.get('port', None): port = ':' + str(pinfo['port']) else: port = '' host = pinfo.get('addr', None) if not host: return proxies all = unpw + host + port proxies = urllib.OneProxy() proxies._proxyServer = all if pinfo['proxytype'] != socks.PROXY_TYPE_HTTP: proxy_url = None % 'socks%d://' if pinfo['proxytype'] == socks.PROXY_TYPE_SOCKS4 else 5 + all return dict(socks = proxy_url, http = proxy_url, https = proxy_url) proxies['https'] = 'http://' + all proxies['http'] = 'http://' + all proxies['ftp'] = 'http://' + all return proxies class SocksProxyHandler(urllib2.ProxyHandler): handler_order = 100 def proxy_open(self, req, type): try: req._proxied except AttributeError: proxyinfo = self.proxies.get(type, '') proxytype = urllib2._parse_proxy(proxyinfo)[0] if proxytype is None: req._proxied = False return urllib2.ProxyHandler.proxy_open(self, req, type) else: req._proxytype = proxytype req._proxied = True if proxytype == 'http' and type != 'https': return urllib2.ProxyHandler.proxy_open(self, req, type) else: return None except: proxytype is None return None def socks4_open(self, req): return self.socks_open(req, 4) def socks5_open(self, req): return self.socks_open(req, 5) def socks_open(self, req, sockstype): (orig_url_type, __, __, orighostport) = urllib2._parse_proxy(req.get_full_url()) req.set_proxy(orighostport, orig_url_type) endpoint = req.get_host() if ':' in endpoint: (host, port) = endpoint.rsplit(':', 1) port = int(port) else: host = endpoint port = 80 req._proxied = True return self.parent.open(req) if hasattr(httplib, 'HTTPS'): class SocksHttpsOpener(urllib2.HTTPSHandler): handler_order = 101 def https_open(self, req): if getattr(req, '_proxied', False) and getattr(req, '_proxytype', None) is not None: return urllib2.HTTPSHandler.do_open(self, SocksHttpsConnection, req) else: return urllib2.HTTPSHandler.https_open(self, req) class SocksHttpsConnection(httplib.HTTPSConnection): _sockettype = socks.socksocket def connect(self): pd = urllib.getproxies().get('https', None) if pd is None: sockstype = '' else: (sockstype, user, password, hostport) = urllib2._parse_proxy(pd) (host, port) = hostport.rsplit(':', 1) port = int(port) sock = self._sockettype(socket.AF_INET, socket.SOCK_STREAM) sock.setproxy(proxytype = getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr = host, port = port, rdns = True, username = user, password = password) sock.connect((self.host, self.port)) ssl = socket.ssl(sock, self.key_file, self.cert_file) self.sock = httplib.FakeSocket(sock, ssl) class SocksHttpOpener(urllib2.HTTPHandler): handler_order = 101 def http_open(self, req): proxytype = getattr(req, '_proxytype', None) if getattr(req, '_proxied', False) and proxytype not in ('http', None): return urllib2.HTTPHandler.do_open(self, SocksConnection, req) else: return urllib2.HTTPHandler.http_open(self, req) class SocksConnection(httplib.HTTPConnection): _sockettype = socks.socksocket def connect(self): pd = urllib.getproxies().get('http', None) if pd is None: sockstype = '' else: (sockstype, user, password, hostport) = urllib2._parse_proxy(pd) if 'socks' not in sockstype: return httplib.HTTPConnection.connect(self) (host, port) = hostport.rsplit(':', 1) port = int(port) for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM): (af, socktype, proto, canonname, sa) = res try: self.sock = self._sockettype(af, socktype, proto) self.sock.setproxy(proxytype = getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr = host, port = port, rdns = False, username = user, password = password) if self.debuglevel > 0: print 'connect: (%s, %s)' % (self.host, self.port) self.sock.connect(sa) except socket.error: msg = None if self.debuglevel > 0: print 'connect fail:', (self.host, self.port) if self.sock: self.sock.close() self.sock = None continue if not self.sock: raise socket.error, msg class DigsbyHttpProxyPasswordManager(urllib2.HTTPPasswordMgr): def find_user_password(self, realm, uri): pi = GetProxyInfo() if not pi['username']: pass if not pi['password']: pass return (None, None) if not hasattr(urllib, '_getproxies'): urllib._getproxies = urllib.getproxies urllib.getproxies = getproxies_digsby urllib2.UnknownHandler.handler_order = sys.maxint urllib2.ProxyDigestAuthHandler.__bases__ = urllib2.ProxyDigestAuthHandler.__bases__[::-1] urllib2.ProxyBasicAuthHandler.handler_order = 499 httplib2.ProxyInfo.get_default_proxy = staticmethod(GetProxyInfoHttp2) def GetDefaultHandlers(): handlers = [ SocksProxyHandler, SocksHttpOpener] httpsopener = globals().get('SocksHttpsOpener', None) if httpsopener is not None: handlers.append(httpsopener) pwdmgr = DigsbyHttpProxyPasswordManager() for auth_handler_type in (urllib2.ProxyBasicAuthHandler, urllib2.ProxyDigestAuthHandler): handlers.append(auth_handler_type(pwdmgr)) return handlers def build_opener(*a, **k): if 'default_classes' not in k: k['default_classes'] = GetDefaultHandlers() return urllib2.build_opener(*a, **k) opener = urllib2.build_opener(*GetDefaultHandlers()) urllib2.install_opener(opener) _hostprog = re.compile('^//([^/?]*)(.*)$') def splithost(url): match = _hostprog.match(url) if match: groups = match.group(1, 2) if groups[0] == '': return (groups[0], '/' + groups[1]) else: return groups return (None, url) urllib.splithost = urllib2.splithost = splithost def httpok(_code): return getattr(_code, 'status', _code) // 100 == 2 class CallbackProducerMixin(object): def __init__(self): bases = self.__class__.__bases__ found_self = False self._siblingClass = None for base in bases: if base is CallbackProducerMixin: found_self = True continue if hasattr(base, 'more') and found_self: self._siblingClass = base break continue if self._siblingClass is None: raise AssertionError("This mix-in requires there is a sibling class with a 'more' method. Additionally, CallbackProducerMixin must be *before* that class in the inheritance list (for method resolution reasons).") def set_callback(self, callback = None): self._callback = callback set_callback = callsback(set_callback) def more(self): if not hasattr(self, '_siblingClass'): result = '' else: result = self._siblingClass.more(self) if result == '': if getattr(self, '_callback', None) is not None: self._callback.success() if getattr(self, '_callback', None) is not None: del self._callback return result class SimpleCallbackProducer(CallbackProducerMixin, asynchat.simple_producer): def __init__(self, data): asynchat.simple_producer.__init__(self, data) CallbackProducerMixin.__init__(self) def _fifo_remove(self, val): try: self.list.remove(val) except Exception: return False return True asynchat.fifo.remove = _fifo_remove def producer_cb(data, callback = None): prod = SimpleCallbackProducer(data) prod.set_callback(callback = callback) return prod producer_cb = callsback(producer_cb) def get_snurl(url): return get_short_url(url, 'snurl') def get_isgd(url): return get_short_url(url, 'isgd') def get_tinyurl(url): return get_short_url(url, 'tinyurl') class UrlShortener(object): endpoint = None def shorten(self, url): try: resp = urllib2.urlopen(UrlQuery(self.endpoint, d = self.get_args(url.encode('utf-8')))) except urllib2.HTTPError: e = None resp = e return self.process_response(resp) def get_args(self, url): raise NotImplementedError def process_response(self, resp): if resp.code != 200: body = resp.read() raise Exception(body) ret = resp.read() return ret class isgd_shortener(UrlShortener): endpoint = 'http://is.gd/api.php' def get_args(self, url): return dict(longurl = url) class tinyurl_shortener(UrlShortener): endpoint = 'http://tinyurl.com/api-create.php' def get_args(self, url): return dict(url = url) class snipr_shortener(UrlShortener): endpoint = 'http://snipr.com/site/snip' def get_args(self, url): return dict(r = 'simple', link = url.encode('url')) def process_response(self, resp): ret = UrlShortener.process_response(self, resp) if not ret.startswith('http'): raise Exception('bad url', ret) return ret _shorteners = { 'snipr': snipr_shortener, 'snurl': snipr_shortener, 'snipurl': snipr_shortener, 'isgd': isgd_shortener, 'tinyurl': tinyurl_shortener, 'tiny': tinyurl_shortener } def get_short_url(url, provider = 'snipr'): shortener = _shorteners.get(provider) if shortener is None: raise Exception('UrlShortener provider %r not found', provider) return shortener().shorten(url) if __name__ == '__main__': print get_snurl('http://www.google.com') 
(.txt)